{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import pickle\n",
    "from torch.utils.data import DataLoader\n",
    "from model.SnliDataSet import SnliDataSet\n",
    "\n",
    "worddict_dir=\"data\\\\worddict.txt\"\n",
    "data_train_id_dir=\"data\\\\train_data_id.pkl\"\n",
    "data_dev_id_dir=\"data\\\\dev_data_id.pkl\"\n",
    "embedding_matrix_dir=\"data\\\\embedding_matrix.pkl\"\n",
    "\n",
    "model_train_dir=\"saved_model\\\\train_model_\"\n",
    "\n",
    "#超参数\n",
    "batch_size=1\n",
    "use_gpu=True\n",
    "patience=5\n",
    "\n",
    "device=torch.device(\"cuda:0\" if use_gpu else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "hidden_size=50\n",
    "dropout=0.5\n",
    "num_classes=3\n",
    "lr=0.0004\n",
    "epochs=2\n",
    "max_grad_norm=10.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 加载数据\n",
    "with open(data_train_id_dir,'rb') as f:\n",
    "    train_data=SnliDataSet(pickle.load(f),max_premises_len=None,max_hypothesis_len=None)\n",
    "train_loader=DataLoader(train_data,batch_size=batch_size,shuffle=True)\n",
    "\n",
    "with open(data_dev_id_dir,'rb') as f:\n",
    "    dev_data=SnliDataSet(pickle.load(f),max_premises_len=None,max_hypothesis_len=None)\n",
    "dev_loader=DataLoader(dev_data,batch_size=batch_size,shuffle=False)\n",
    "\n",
    "#加载embedding\n",
    "with open(embedding_matrix_dir,'rb') as f:\n",
    "    embeddings=torch.tensor(pickle.load(f),dtype=torch.float).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from model.esim import ESIM\n",
    "model = ESIM(embeddings.shape[0],\n",
    "             embeddings.shape[1],\n",
    "             hidden_size,\n",
    "             embeddings=embeddings,\n",
    "             dropout=dropout,\n",
    "             num_classes=num_classes,\n",
    "             device=device).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#准备训练\n",
    "criterion = torch.nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n",
    "scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,mode=\"max\",factor=0.5,patience=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getCorrectNum(probs, targets):\n",
    "    _, out_classes = probs.max(dim=1)\n",
    "    correct = (out_classes == targets).sum()\n",
    "    return correct.item()\n",
    "\n",
    "def train(model, data_loader, optimizer, criterion, max_gradient_norm):\n",
    "    model.train()\n",
    "    device=model.device\n",
    "    \n",
    "    time_epoch_start= time.time()\n",
    "    running_loss=0 \n",
    "    correct_cnt=0\n",
    "    batch_cnt=0\n",
    "    \n",
    "    for index,batch in enumerate(data_loader):\n",
    "        time_batch_start=time.time()\n",
    "        #从data_loader中取出数据\n",
    "        premises=batch[\"premises\"].to(device)\n",
    "        premises_len=batch[\"premises_len\"].to(device)\n",
    "        hypothesis=batch[\"hypothesis\"].to(device)\n",
    "        hypothesis_len=batch[\"hypothesis_len\"].to(device)\n",
    "        labels=batch[\"labels\"].to(device)\n",
    "        \n",
    "        #梯度置0\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        #正向传播\n",
    "        logits,probs=model(premises,premises_len,hypothesis,hypothesis_len)\n",
    "\n",
    "        #求损失，反向传播，梯度裁剪，更新权重\n",
    "        loss = criterion(logits, labels)\n",
    "        loss.backward()\n",
    "        nn.utils.clip_grad_norm_(model.parameters(), max_gradient_norm)\n",
    "        optimizer.step()\n",
    "        \n",
    "        running_loss+=loss.item()\n",
    "        correct_cnt+=getCorrectNum(probs,labels)\n",
    "        batch_cnt+=1\n",
    "        print(\"Training  ------>   Batch count: {:d}/{:d},  batch time: {:.4f}s,  batch average loss: {:.4f}\"\n",
    "              .format(batch_cnt,len(data_loader),time.time()-time_batch_start, running_loss/(index+1)))\n",
    "        \n",
    "    epoch_time = time.time() - time_epoch_start\n",
    "    epoch_loss = running_loss / len(data_loader)\n",
    "    epoch_accuracy = correct_cnt / len(data_loader.dataset) \n",
    "    return epoch_time,epoch_loss,epoch_accuracy\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "def validate(model, data_loader, criterion):\n",
    "    model.eval()\n",
    "    device=model.device\n",
    "    \n",
    "    time_epoch_start= time.time()\n",
    "    running_loss=0 \n",
    "    correct_cnt=0\n",
    "    batch_cnt=0\n",
    "\n",
    "    for index,batch in enumerate(data_loader):\n",
    "        time_batch_start=time.time()\n",
    "        #从data_loader中取出数据\n",
    "        premises=batch[\"premises\"].to(device)\n",
    "        premises_len=batch[\"premises_len\"].to(device)\n",
    "        hypothesis=batch[\"hypothesis\"].to(device)\n",
    "        hypothesis_len=batch[\"hypothesis_len\"].to(device)\n",
    "        labels=batch[\"labels\"].to(device)\n",
    "        \n",
    "        \n",
    "        #正向传播\n",
    "        logits,probs=model(premises,premises_len,hypothesis,hypothesis_len)\n",
    "\n",
    "        #求损失\n",
    "        loss = criterion(logits, labels)\n",
    "        \n",
    "        running_loss+=loss.item()\n",
    "        correct_cnt+=getCorrectNum(probs,labels)\n",
    "        batch_cnt+=1\n",
    "        print(\"Testing  ------>   Batch count: {:d}/{:d},  batch time: {:.4f}s,  batch average loss: {:.4f}\"\n",
    "              .format(batch_cnt,len(data_loader),time.time()-time_batch_start, running_loss/(index+1)))\n",
    "        \n",
    "    epoch_time = time.time() - time_epoch_start\n",
    "    epoch_loss = running_loss / len(data_loader)\n",
    "    epoch_accuracy = correct_cnt / len(data_loader.dataset) \n",
    "    return epoch_time,epoch_loss,epoch_accuracy\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------------------- Training epoch 0 --------------------------------------------------\n",
      "Training  ------>   Batch count: 1/549367,  batch time: 0.2719s,  batch average loss: 1.3751\n",
      "Training  ------>   Batch count: 2/549367,  batch time: 0.0379s,  batch average loss: 1.2290\n",
      "Training  ------>   Batch count: 3/549367,  batch time: 0.0409s,  batch average loss: 1.2561\n",
      "Training  ------>   Batch count: 4/549367,  batch time: 0.0389s,  batch average loss: 1.1968\n",
      "Training  ------>   Batch count: 5/549367,  batch time: 0.0481s,  batch average loss: 1.1857\n",
      "Training  ------>   Batch count: 6/549367,  batch time: 0.0382s,  batch average loss: 1.1873\n",
      "Training  ------>   Batch count: 7/549367,  batch time: 0.0389s,  batch average loss: 1.1692\n",
      "Training  ------>   Batch count: 8/549367,  batch time: 0.0366s,  batch average loss: 1.1703\n",
      "Training  ------>   Batch count: 9/549367,  batch time: 0.0387s,  batch average loss: 1.1602\n",
      "Training  ------>   Batch count: 10/549367,  batch time: 0.0400s,  batch average loss: 1.1461\n",
      "Training  ------>   Batch count: 11/549367,  batch time: 0.0358s,  batch average loss: 1.1340\n",
      "Training  ------>   Batch count: 12/549367,  batch time: 0.0391s,  batch average loss: 1.1469\n",
      "Training  ------>   Batch count: 13/549367,  batch time: 0.0384s,  batch average loss: 1.1738\n",
      "Training  ------>   Batch count: 14/549367,  batch time: 0.0451s,  batch average loss: 1.1757\n",
      "Training  ------>   Batch count: 15/549367,  batch time: 0.0414s,  batch average loss: 1.1707\n",
      "Training  ------>   Batch count: 16/549367,  batch time: 0.0417s,  batch average loss: 1.1665\n",
      "Training  ------>   Batch count: 17/549367,  batch time: 0.0394s,  batch average loss: 1.1610\n",
      "Training  ------>   Batch count: 18/549367,  batch time: 0.0406s,  batch average loss: 1.1561\n",
      "Training  ------>   Batch count: 19/549367,  batch time: 0.0440s,  batch average loss: 1.1703\n",
      "Training  ------>   Batch count: 20/549367,  batch time: 0.0405s,  batch average loss: 1.1685\n",
      "Training  ------>   Batch count: 21/549367,  batch time: 0.0411s,  batch average loss: 1.1626\n",
      "Training  ------>   Batch count: 22/549367,  batch time: 0.0379s,  batch average loss: 1.1672\n",
      "Training  ------>   Batch count: 23/549367,  batch time: 0.0376s,  batch average loss: 1.1725\n",
      "Training  ------>   Batch count: 24/549367,  batch time: 0.0409s,  batch average loss: 1.1667\n",
      "Training  ------>   Batch count: 25/549367,  batch time: 0.0421s,  batch average loss: 1.1545\n",
      "Training  ------>   Batch count: 26/549367,  batch time: 0.0434s,  batch average loss: 1.1490\n",
      "Training  ------>   Batch count: 27/549367,  batch time: 0.0456s,  batch average loss: 1.1501\n",
      "Training  ------>   Batch count: 28/549367,  batch time: 0.0375s,  batch average loss: 1.1505\n",
      "Training  ------>   Batch count: 29/549367,  batch time: 0.0393s,  batch average loss: 1.1490\n",
      "Training  ------>   Batch count: 30/549367,  batch time: 0.0393s,  batch average loss: 1.1482\n",
      "Training  ------>   Batch count: 31/549367,  batch time: 0.0395s,  batch average loss: 1.1493\n",
      "Training  ------>   Batch count: 32/549367,  batch time: 0.0363s,  batch average loss: 1.1485\n",
      "Training  ------>   Batch count: 33/549367,  batch time: 0.0377s,  batch average loss: 1.1479\n",
      "Training  ------>   Batch count: 34/549367,  batch time: 0.0425s,  batch average loss: 1.1517\n",
      "Training  ------>   Batch count: 35/549367,  batch time: 0.0385s,  batch average loss: 1.1553\n",
      "Training  ------>   Batch count: 36/549367,  batch time: 0.0386s,  batch average loss: 1.1608\n",
      "Training  ------>   Batch count: 37/549367,  batch time: 0.0399s,  batch average loss: 1.1557\n",
      "Training  ------>   Batch count: 38/549367,  batch time: 0.0373s,  batch average loss: 1.1539\n",
      "Training  ------>   Batch count: 39/549367,  batch time: 0.0421s,  batch average loss: 1.1515\n",
      "Training  ------>   Batch count: 40/549367,  batch time: 0.0400s,  batch average loss: 1.1493\n",
      "Training  ------>   Batch count: 41/549367,  batch time: 0.0414s,  batch average loss: 1.1583\n",
      "Training  ------>   Batch count: 42/549367,  batch time: 0.0427s,  batch average loss: 1.1566\n",
      "Training  ------>   Batch count: 43/549367,  batch time: 0.0451s,  batch average loss: 1.1513\n",
      "Training  ------>   Batch count: 44/549367,  batch time: 0.0392s,  batch average loss: 1.1458\n",
      "Training  ------>   Batch count: 45/549367,  batch time: 0.0359s,  batch average loss: 1.1460\n",
      "Training  ------>   Batch count: 46/549367,  batch time: 0.0413s,  batch average loss: 1.1468\n",
      "Training  ------>   Batch count: 47/549367,  batch time: 0.0428s,  batch average loss: 1.1456\n",
      "Training  ------>   Batch count: 48/549367,  batch time: 0.0432s,  batch average loss: 1.1415\n",
      "Training  ------>   Batch count: 49/549367,  batch time: 0.0396s,  batch average loss: 1.1384\n",
      "Training  ------>   Batch count: 50/549367,  batch time: 0.0367s,  batch average loss: 1.1387\n",
      "Training  ------>   Batch count: 51/549367,  batch time: 0.0391s,  batch average loss: 1.1365\n",
      "Training  ------>   Batch count: 52/549367,  batch time: 0.0423s,  batch average loss: 1.1371\n",
      "Training  ------>   Batch count: 53/549367,  batch time: 0.0406s,  batch average loss: 1.1361\n",
      "Training  ------>   Batch count: 54/549367,  batch time: 0.0379s,  batch average loss: 1.1366\n",
      "Training  ------>   Batch count: 55/549367,  batch time: 0.0488s,  batch average loss: 1.1364\n",
      "Training  ------>   Batch count: 56/549367,  batch time: 0.0469s,  batch average loss: 1.1353\n",
      "Training  ------>   Batch count: 57/549367,  batch time: 0.0431s,  batch average loss: 1.1329\n",
      "Training  ------>   Batch count: 58/549367,  batch time: 0.0380s,  batch average loss: 1.1302\n",
      "Training  ------>   Batch count: 59/549367,  batch time: 0.0396s,  batch average loss: 1.1305\n",
      "Training  ------>   Batch count: 60/549367,  batch time: 0.0418s,  batch average loss: 1.1293\n",
      "Training  ------>   Batch count: 61/549367,  batch time: 0.0385s,  batch average loss: 1.1273\n",
      "Training  ------>   Batch count: 62/549367,  batch time: 0.0400s,  batch average loss: 1.1265\n",
      "Training  ------>   Batch count: 63/549367,  batch time: 0.0367s,  batch average loss: 1.1262\n",
      "Training  ------>   Batch count: 64/549367,  batch time: 0.0468s,  batch average loss: 1.1298\n",
      "Training  ------>   Batch count: 65/549367,  batch time: 0.0395s,  batch average loss: 1.1283\n",
      "Training  ------>   Batch count: 66/549367,  batch time: 0.0408s,  batch average loss: 1.1268\n",
      "Training  ------>   Batch count: 67/549367,  batch time: 0.0370s,  batch average loss: 1.1256\n",
      "Training  ------>   Batch count: 68/549367,  batch time: 0.0389s,  batch average loss: 1.1253\n",
      "Training  ------>   Batch count: 69/549367,  batch time: 0.0368s,  batch average loss: 1.1261\n",
      "Training  ------>   Batch count: 70/549367,  batch time: 0.0417s,  batch average loss: 1.1259\n",
      "Training  ------>   Batch count: 71/549367,  batch time: 0.0409s,  batch average loss: 1.1255\n",
      "Training  ------>   Batch count: 72/549367,  batch time: 0.0377s,  batch average loss: 1.1234\n",
      "Training  ------>   Batch count: 73/549367,  batch time: 0.0399s,  batch average loss: 1.1224\n",
      "Training  ------>   Batch count: 74/549367,  batch time: 0.0401s,  batch average loss: 1.1223\n",
      "Training  ------>   Batch count: 75/549367,  batch time: 0.0390s,  batch average loss: 1.1211\n",
      "Training  ------>   Batch count: 76/549367,  batch time: 0.0371s,  batch average loss: 1.1239\n",
      "Training  ------>   Batch count: 77/549367,  batch time: 0.0410s,  batch average loss: 1.1202\n",
      "Training  ------>   Batch count: 78/549367,  batch time: 0.0449s,  batch average loss: 1.1182\n",
      "Training  ------>   Batch count: 79/549367,  batch time: 0.0457s,  batch average loss: 1.1191\n",
      "Training  ------>   Batch count: 80/549367,  batch time: 0.0393s,  batch average loss: 1.1189\n",
      "Training  ------>   Batch count: 81/549367,  batch time: 0.0381s,  batch average loss: 1.1190\n",
      "Training  ------>   Batch count: 82/549367,  batch time: 0.0443s,  batch average loss: 1.1193\n",
      "Training  ------>   Batch count: 83/549367,  batch time: 0.0386s,  batch average loss: 1.1194\n",
      "Training  ------>   Batch count: 84/549367,  batch time: 0.0379s,  batch average loss: 1.1179\n",
      "Training  ------>   Batch count: 85/549367,  batch time: 0.0385s,  batch average loss: 1.1190\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training  ------>   Batch count: 86/549367,  batch time: 0.0380s,  batch average loss: 1.1175\n",
      "Training  ------>   Batch count: 87/549367,  batch time: 0.0416s,  batch average loss: 1.1164\n",
      "Training  ------>   Batch count: 88/549367,  batch time: 0.0379s,  batch average loss: 1.1160\n",
      "Training  ------>   Batch count: 89/549367,  batch time: 0.0417s,  batch average loss: 1.1155\n",
      "Training  ------>   Batch count: 90/549367,  batch time: 0.0435s,  batch average loss: 1.1155\n",
      "Training  ------>   Batch count: 91/549367,  batch time: 0.0521s,  batch average loss: 1.1147\n",
      "Training  ------>   Batch count: 92/549367,  batch time: 0.0417s,  batch average loss: 1.1151\n",
      "Training  ------>   Batch count: 93/549367,  batch time: 0.0396s,  batch average loss: 1.1154\n",
      "Training  ------>   Batch count: 94/549367,  batch time: 0.0387s,  batch average loss: 1.1148\n",
      "Training  ------>   Batch count: 95/549367,  batch time: 0.0428s,  batch average loss: 1.1147\n",
      "Training  ------>   Batch count: 96/549367,  batch time: 0.0385s,  batch average loss: 1.1144\n",
      "Training  ------>   Batch count: 97/549367,  batch time: 0.0392s,  batch average loss: 1.1142\n",
      "Training  ------>   Batch count: 98/549367,  batch time: 0.0360s,  batch average loss: 1.1158\n",
      "Training  ------>   Batch count: 99/549367,  batch time: 0.0411s,  batch average loss: 1.1150\n",
      "Training  ------>   Batch count: 100/549367,  batch time: 0.0385s,  batch average loss: 1.1128\n",
      "Training  ------>   Batch count: 101/549367,  batch time: 0.0406s,  batch average loss: 1.1128\n",
      "Training time: 4.5070s, loss :0.0002, accuracy: 0.0064%\n",
      "-------------------------------------------------- Validating epoch 0 --------------------------------------------------\n",
      "Testing  ------>   Batch count: 1/9842,  batch time: 0.0179s,  batch average loss: 1.0848\n",
      "Testing  ------>   Batch count: 2/9842,  batch time: 0.0142s,  batch average loss: 1.0843\n",
      "Testing  ------>   Batch count: 3/9842,  batch time: 0.0184s,  batch average loss: 1.0877\n",
      "Testing  ------>   Batch count: 4/9842,  batch time: 0.0157s,  batch average loss: 1.0887\n",
      "Testing  ------>   Batch count: 5/9842,  batch time: 0.0186s,  batch average loss: 1.0938\n",
      "Testing  ------>   Batch count: 6/9842,  batch time: 0.0132s,  batch average loss: 1.0947\n",
      "Testing  ------>   Batch count: 7/9842,  batch time: 0.0159s,  batch average loss: 1.0943\n",
      "Testing  ------>   Batch count: 8/9842,  batch time: 0.0131s,  batch average loss: 1.0929\n",
      "Testing  ------>   Batch count: 9/9842,  batch time: 0.0130s,  batch average loss: 1.0922\n",
      "Testing  ------>   Batch count: 10/9842,  batch time: 0.0129s,  batch average loss: 1.0882\n",
      "Testing  ------>   Batch count: 11/9842,  batch time: 0.0150s,  batch average loss: 1.0881\n",
      "Testing  ------>   Batch count: 12/9842,  batch time: 0.0131s,  batch average loss: 1.0945\n",
      "Testing  ------>   Batch count: 13/9842,  batch time: 0.0141s,  batch average loss: 1.0964\n",
      "Testing  ------>   Batch count: 14/9842,  batch time: 0.0130s,  batch average loss: 1.0959\n",
      "Testing  ------>   Batch count: 15/9842,  batch time: 0.0130s,  batch average loss: 1.0958\n",
      "Testing  ------>   Batch count: 16/9842,  batch time: 0.0137s,  batch average loss: 1.0965\n",
      "Testing  ------>   Batch count: 17/9842,  batch time: 0.0136s,  batch average loss: 1.0964\n",
      "Testing  ------>   Batch count: 18/9842,  batch time: 0.0149s,  batch average loss: 1.0965\n",
      "Testing  ------>   Batch count: 19/9842,  batch time: 0.0126s,  batch average loss: 1.0951\n",
      "Testing  ------>   Batch count: 20/9842,  batch time: 0.0130s,  batch average loss: 1.0951\n",
      "Testing  ------>   Batch count: 21/9842,  batch time: 0.0172s,  batch average loss: 1.0949\n",
      "Testing  ------>   Batch count: 22/9842,  batch time: 0.0144s,  batch average loss: 1.0938\n",
      "Testing  ------>   Batch count: 23/9842,  batch time: 0.0155s,  batch average loss: 1.0944\n",
      "Testing  ------>   Batch count: 24/9842,  batch time: 0.0142s,  batch average loss: 1.0942\n",
      "Testing  ------>   Batch count: 25/9842,  batch time: 0.0156s,  batch average loss: 1.0953\n",
      "Testing  ------>   Batch count: 26/9842,  batch time: 0.0139s,  batch average loss: 1.0940\n",
      "Testing  ------>   Batch count: 27/9842,  batch time: 0.0140s,  batch average loss: 1.0943\n",
      "Testing  ------>   Batch count: 28/9842,  batch time: 0.0155s,  batch average loss: 1.0946\n",
      "Testing  ------>   Batch count: 29/9842,  batch time: 0.0168s,  batch average loss: 1.0946\n",
      "Testing  ------>   Batch count: 30/9842,  batch time: 0.0180s,  batch average loss: 1.0947\n",
      "Testing  ------>   Batch count: 31/9842,  batch time: 0.0156s,  batch average loss: 1.0949\n",
      "Testing  ------>   Batch count: 32/9842,  batch time: 0.0154s,  batch average loss: 1.0947\n",
      "Testing  ------>   Batch count: 33/9842,  batch time: 0.0166s,  batch average loss: 1.0948\n",
      "Testing  ------>   Batch count: 34/9842,  batch time: 0.0140s,  batch average loss: 1.0942\n",
      "Testing  ------>   Batch count: 35/9842,  batch time: 0.0170s,  batch average loss: 1.0942\n",
      "Testing  ------>   Batch count: 36/9842,  batch time: 0.0150s,  batch average loss: 1.0951\n",
      "Testing  ------>   Batch count: 37/9842,  batch time: 0.0135s,  batch average loss: 1.0949\n",
      "Testing  ------>   Batch count: 38/9842,  batch time: 0.0150s,  batch average loss: 1.0949\n",
      "Testing  ------>   Batch count: 39/9842,  batch time: 0.0157s,  batch average loss: 1.0955\n",
      "Testing  ------>   Batch count: 40/9842,  batch time: 0.0157s,  batch average loss: 1.0955\n",
      "Testing  ------>   Batch count: 41/9842,  batch time: 0.0131s,  batch average loss: 1.0966\n",
      "Testing  ------>   Batch count: 42/9842,  batch time: 0.0140s,  batch average loss: 1.0969\n",
      "Testing  ------>   Batch count: 43/9842,  batch time: 0.0135s,  batch average loss: 1.0970\n",
      "Testing  ------>   Batch count: 44/9842,  batch time: 0.0130s,  batch average loss: 1.0963\n",
      "Testing  ------>   Batch count: 45/9842,  batch time: 0.0152s,  batch average loss: 1.0964\n",
      "Testing  ------>   Batch count: 46/9842,  batch time: 0.0152s,  batch average loss: 1.0964\n",
      "Testing  ------>   Batch count: 47/9842,  batch time: 0.0126s,  batch average loss: 1.0965\n",
      "Testing  ------>   Batch count: 48/9842,  batch time: 0.0140s,  batch average loss: 1.0960\n",
      "Testing  ------>   Batch count: 49/9842,  batch time: 0.0130s,  batch average loss: 1.0964\n",
      "Testing  ------>   Batch count: 50/9842,  batch time: 0.0120s,  batch average loss: 1.0962\n",
      "Testing  ------>   Batch count: 51/9842,  batch time: 0.0120s,  batch average loss: 1.0956\n",
      "Testing  ------>   Batch count: 52/9842,  batch time: 0.0164s,  batch average loss: 1.0957\n",
      "Testing  ------>   Batch count: 53/9842,  batch time: 0.0150s,  batch average loss: 1.0954\n",
      "Testing  ------>   Batch count: 54/9842,  batch time: 0.0140s,  batch average loss: 1.0959\n",
      "Testing  ------>   Batch count: 55/9842,  batch time: 0.0130s,  batch average loss: 1.0964\n",
      "Testing  ------>   Batch count: 56/9842,  batch time: 0.0140s,  batch average loss: 1.0960\n",
      "Testing  ------>   Batch count: 57/9842,  batch time: 0.0149s,  batch average loss: 1.0959\n",
      "Testing  ------>   Batch count: 58/9842,  batch time: 0.0160s,  batch average loss: 1.0959\n",
      "Testing  ------>   Batch count: 59/9842,  batch time: 0.0160s,  batch average loss: 1.0959\n",
      "Testing  ------>   Batch count: 60/9842,  batch time: 0.0169s,  batch average loss: 1.0959\n",
      "Testing  ------>   Batch count: 61/9842,  batch time: 0.0140s,  batch average loss: 1.0959\n",
      "Testing  ------>   Batch count: 62/9842,  batch time: 0.0150s,  batch average loss: 1.0962\n",
      "Testing  ------>   Batch count: 63/9842,  batch time: 0.0140s,  batch average loss: 1.0958\n",
      "Testing  ------>   Batch count: 64/9842,  batch time: 0.0146s,  batch average loss: 1.0958\n",
      "Testing  ------>   Batch count: 65/9842,  batch time: 0.0128s,  batch average loss: 1.0961\n",
      "Testing  ------>   Batch count: 66/9842,  batch time: 0.0133s,  batch average loss: 1.0960\n",
      "Testing  ------>   Batch count: 67/9842,  batch time: 0.0126s,  batch average loss: 1.0959\n",
      "Testing  ------>   Batch count: 68/9842,  batch time: 0.0140s,  batch average loss: 1.0960\n",
      "Testing  ------>   Batch count: 69/9842,  batch time: 0.0170s,  batch average loss: 1.0959\n",
      "Testing  ------>   Batch count: 70/9842,  batch time: 0.0146s,  batch average loss: 1.0959\n",
      "Testing  ------>   Batch count: 71/9842,  batch time: 0.0141s,  batch average loss: 1.0958\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing  ------>   Batch count: 72/9842,  batch time: 0.0130s,  batch average loss: 1.0959\n",
      "Testing  ------>   Batch count: 73/9842,  batch time: 0.0156s,  batch average loss: 1.0955\n",
      "Testing  ------>   Batch count: 74/9842,  batch time: 0.0160s,  batch average loss: 1.0959\n",
      "Testing  ------>   Batch count: 75/9842,  batch time: 0.0140s,  batch average loss: 1.0958\n",
      "Testing  ------>   Batch count: 76/9842,  batch time: 0.0146s,  batch average loss: 1.0957\n",
      "Testing  ------>   Batch count: 77/9842,  batch time: 0.0136s,  batch average loss: 1.0958\n",
      "Testing  ------>   Batch count: 78/9842,  batch time: 0.0125s,  batch average loss: 1.0965\n",
      "Testing  ------>   Batch count: 79/9842,  batch time: 0.0136s,  batch average loss: 1.0964\n",
      "Testing  ------>   Batch count: 80/9842,  batch time: 0.0146s,  batch average loss: 1.0963\n",
      "Testing  ------>   Batch count: 81/9842,  batch time: 0.0134s,  batch average loss: 1.0970\n",
      "Testing  ------>   Batch count: 82/9842,  batch time: 0.0145s,  batch average loss: 1.0969\n",
      "Testing  ------>   Batch count: 83/9842,  batch time: 0.0160s,  batch average loss: 1.0969\n",
      "Testing  ------>   Batch count: 84/9842,  batch time: 0.0120s,  batch average loss: 1.0969\n",
      "Testing  ------>   Batch count: 85/9842,  batch time: 0.0125s,  batch average loss: 1.0967\n",
      "Testing  ------>   Batch count: 86/9842,  batch time: 0.0135s,  batch average loss: 1.0967\n",
      "Testing  ------>   Batch count: 87/9842,  batch time: 0.0119s,  batch average loss: 1.0968\n",
      "Testing  ------>   Batch count: 88/9842,  batch time: 0.0139s,  batch average loss: 1.0971\n",
      "Testing  ------>   Batch count: 89/9842,  batch time: 0.0129s,  batch average loss: 1.0970\n",
      "Testing  ------>   Batch count: 90/9842,  batch time: 0.0137s,  batch average loss: 1.0970\n",
      "Testing  ------>   Batch count: 91/9842,  batch time: 0.0142s,  batch average loss: 1.0969\n",
      "Testing  ------>   Batch count: 92/9842,  batch time: 0.0156s,  batch average loss: 1.0970\n",
      "Testing  ------>   Batch count: 93/9842,  batch time: 0.0159s,  batch average loss: 1.0972\n",
      "Testing  ------>   Batch count: 94/9842,  batch time: 0.0146s,  batch average loss: 1.0972\n",
      "Testing  ------>   Batch count: 95/9842,  batch time: 0.0146s,  batch average loss: 1.0970\n",
      "Testing  ------>   Batch count: 96/9842,  batch time: 0.0136s,  batch average loss: 1.0966\n",
      "Testing  ------>   Batch count: 97/9842,  batch time: 0.0136s,  batch average loss: 1.0966\n",
      "Testing  ------>   Batch count: 98/9842,  batch time: 0.0169s,  batch average loss: 1.0962\n",
      "Testing  ------>   Batch count: 99/9842,  batch time: 0.0129s,  batch average loss: 1.0962\n",
      "Testing  ------>   Batch count: 100/9842,  batch time: 0.0124s,  batch average loss: 1.0962\n",
      "Testing  ------>   Batch count: 101/9842,  batch time: 0.0130s,  batch average loss: 1.0961\n",
      "Validating time: 1.6126s, loss: 0.0112, accuracy: 0.4166%\n",
      "\n",
      "-------------------------------------------------- Training epoch 1 --------------------------------------------------\n",
      "Training  ------>   Batch count: 1/549367,  batch time: 0.0415s,  batch average loss: 1.0007\n",
      "Training  ------>   Batch count: 2/549367,  batch time: 0.0378s,  batch average loss: 1.0168\n",
      "Training  ------>   Batch count: 3/549367,  batch time: 0.0388s,  batch average loss: 1.0309\n",
      "Training  ------>   Batch count: 4/549367,  batch time: 0.0447s,  batch average loss: 1.0630\n",
      "Training  ------>   Batch count: 5/549367,  batch time: 0.0387s,  batch average loss: 1.0502\n",
      "Training  ------>   Batch count: 6/549367,  batch time: 0.0455s,  batch average loss: 1.0703\n",
      "Training  ------>   Batch count: 7/549367,  batch time: 0.0384s,  batch average loss: 1.0628\n",
      "Training  ------>   Batch count: 8/549367,  batch time: 0.0362s,  batch average loss: 1.0710\n",
      "Training  ------>   Batch count: 9/549367,  batch time: 0.0445s,  batch average loss: 1.0709\n",
      "Training  ------>   Batch count: 10/549367,  batch time: 0.0383s,  batch average loss: 1.0407\n",
      "Training  ------>   Batch count: 11/549367,  batch time: 0.0453s,  batch average loss: 1.0530\n",
      "Training  ------>   Batch count: 12/549367,  batch time: 0.0379s,  batch average loss: 1.0431\n",
      "Training  ------>   Batch count: 13/549367,  batch time: 0.0378s,  batch average loss: 1.0406\n",
      "Training  ------>   Batch count: 14/549367,  batch time: 0.0361s,  batch average loss: 1.0428\n",
      "Training  ------>   Batch count: 15/549367,  batch time: 0.0441s,  batch average loss: 1.0450\n",
      "Training  ------>   Batch count: 16/549367,  batch time: 0.0375s,  batch average loss: 1.0446\n",
      "Training  ------>   Batch count: 17/549367,  batch time: 0.0395s,  batch average loss: 1.0328\n",
      "Training  ------>   Batch count: 18/549367,  batch time: 0.0424s,  batch average loss: 1.0357\n",
      "Training  ------>   Batch count: 19/549367,  batch time: 0.0394s,  batch average loss: 1.0465\n",
      "Training  ------>   Batch count: 20/549367,  batch time: 0.0389s,  batch average loss: 1.0518\n",
      "Training  ------>   Batch count: 21/549367,  batch time: 0.0430s,  batch average loss: 1.0554\n",
      "Training  ------>   Batch count: 22/549367,  batch time: 0.0484s,  batch average loss: 1.0556\n",
      "Training  ------>   Batch count: 23/549367,  batch time: 0.0378s,  batch average loss: 1.0615\n",
      "Training  ------>   Batch count: 24/549367,  batch time: 0.0369s,  batch average loss: 1.0609\n",
      "Training  ------>   Batch count: 25/549367,  batch time: 0.0399s,  batch average loss: 1.0632\n",
      "Training  ------>   Batch count: 26/549367,  batch time: 0.0405s,  batch average loss: 1.0679\n",
      "Training  ------>   Batch count: 27/549367,  batch time: 0.0500s,  batch average loss: 1.0666\n",
      "Training  ------>   Batch count: 28/549367,  batch time: 0.0450s,  batch average loss: 1.0690\n",
      "Training  ------>   Batch count: 29/549367,  batch time: 0.0409s,  batch average loss: 1.0706\n",
      "Training  ------>   Batch count: 30/549367,  batch time: 0.0351s,  batch average loss: 1.0706\n",
      "Training  ------>   Batch count: 31/549367,  batch time: 0.0380s,  batch average loss: 1.0696\n",
      "Training  ------>   Batch count: 32/549367,  batch time: 0.0365s,  batch average loss: 1.0720\n",
      "Training  ------>   Batch count: 33/549367,  batch time: 0.0356s,  batch average loss: 1.0806\n",
      "Training  ------>   Batch count: 34/549367,  batch time: 0.0361s,  batch average loss: 1.0843\n",
      "Training  ------>   Batch count: 35/549367,  batch time: 0.0391s,  batch average loss: 1.0870\n",
      "Training  ------>   Batch count: 36/549367,  batch time: 0.0364s,  batch average loss: 1.0872\n",
      "Training  ------>   Batch count: 37/549367,  batch time: 0.0387s,  batch average loss: 1.0866\n",
      "Training  ------>   Batch count: 38/549367,  batch time: 0.0366s,  batch average loss: 1.0919\n",
      "Training  ------>   Batch count: 39/549367,  batch time: 0.0398s,  batch average loss: 1.0965\n",
      "Training  ------>   Batch count: 40/549367,  batch time: 0.0394s,  batch average loss: 1.0982\n",
      "Training  ------>   Batch count: 41/549367,  batch time: 0.0378s,  batch average loss: 1.0967\n",
      "Training  ------>   Batch count: 42/549367,  batch time: 0.0379s,  batch average loss: 1.0977\n",
      "Training  ------>   Batch count: 43/549367,  batch time: 0.0369s,  batch average loss: 1.0954\n",
      "Training  ------>   Batch count: 44/549367,  batch time: 0.0380s,  batch average loss: 1.0963\n",
      "Training  ------>   Batch count: 45/549367,  batch time: 0.0356s,  batch average loss: 1.0956\n",
      "Training  ------>   Batch count: 46/549367,  batch time: 0.0429s,  batch average loss: 1.0949\n",
      "Training  ------>   Batch count: 47/549367,  batch time: 0.0440s,  batch average loss: 1.0905\n",
      "Training  ------>   Batch count: 48/549367,  batch time: 0.0394s,  batch average loss: 1.0919\n",
      "Training  ------>   Batch count: 49/549367,  batch time: 0.0387s,  batch average loss: 1.0916\n",
      "Training  ------>   Batch count: 50/549367,  batch time: 0.0399s,  batch average loss: 1.0911\n",
      "Training  ------>   Batch count: 51/549367,  batch time: 0.0361s,  batch average loss: 1.0909\n",
      "Training  ------>   Batch count: 52/549367,  batch time: 0.0369s,  batch average loss: 1.0914\n",
      "Training  ------>   Batch count: 53/549367,  batch time: 0.0409s,  batch average loss: 1.0894\n",
      "Training  ------>   Batch count: 54/549367,  batch time: 0.0370s,  batch average loss: 1.0896\n",
      "Training  ------>   Batch count: 55/549367,  batch time: 0.0356s,  batch average loss: 1.0864\n",
      "Training  ------>   Batch count: 56/549367,  batch time: 0.0425s,  batch average loss: 1.0853\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training  ------>   Batch count: 57/549367,  batch time: 0.0479s,  batch average loss: 1.0853\n",
      "Training  ------>   Batch count: 58/549367,  batch time: 0.0378s,  batch average loss: 1.0856\n",
      "Training  ------>   Batch count: 59/549367,  batch time: 0.0415s,  batch average loss: 1.0859\n",
      "Training  ------>   Batch count: 60/549367,  batch time: 0.0392s,  batch average loss: 1.0871\n",
      "Training  ------>   Batch count: 61/549367,  batch time: 0.0381s,  batch average loss: 1.0860\n",
      "Training  ------>   Batch count: 62/549367,  batch time: 0.0378s,  batch average loss: 1.0852\n",
      "Training  ------>   Batch count: 63/549367,  batch time: 0.0407s,  batch average loss: 1.0847\n",
      "Training  ------>   Batch count: 64/549367,  batch time: 0.0365s,  batch average loss: 1.0843\n",
      "Training  ------>   Batch count: 65/549367,  batch time: 0.0417s,  batch average loss: 1.0850\n",
      "Training  ------>   Batch count: 66/549367,  batch time: 0.0403s,  batch average loss: 1.0829\n",
      "Training  ------>   Batch count: 67/549367,  batch time: 0.0420s,  batch average loss: 1.0816\n",
      "Training  ------>   Batch count: 68/549367,  batch time: 0.0361s,  batch average loss: 1.0810\n",
      "Training  ------>   Batch count: 69/549367,  batch time: 0.0418s,  batch average loss: 1.0809\n",
      "Training  ------>   Batch count: 70/549367,  batch time: 0.0375s,  batch average loss: 1.0852\n",
      "Training  ------>   Batch count: 71/549367,  batch time: 0.0419s,  batch average loss: 1.0863\n",
      "Training  ------>   Batch count: 72/549367,  batch time: 0.0429s,  batch average loss: 1.0860\n",
      "Training  ------>   Batch count: 73/549367,  batch time: 0.0415s,  batch average loss: 1.0846\n",
      "Training  ------>   Batch count: 74/549367,  batch time: 0.0391s,  batch average loss: 1.0875\n",
      "Training  ------>   Batch count: 75/549367,  batch time: 0.0405s,  batch average loss: 1.0857\n",
      "Training  ------>   Batch count: 76/549367,  batch time: 0.0389s,  batch average loss: 1.0871\n",
      "Training  ------>   Batch count: 77/549367,  batch time: 0.0375s,  batch average loss: 1.0862\n",
      "Training  ------>   Batch count: 78/549367,  batch time: 0.0423s,  batch average loss: 1.0877\n",
      "Training  ------>   Batch count: 79/549367,  batch time: 0.0369s,  batch average loss: 1.0876\n",
      "Training  ------>   Batch count: 80/549367,  batch time: 0.0408s,  batch average loss: 1.0873\n",
      "Training  ------>   Batch count: 81/549367,  batch time: 0.0398s,  batch average loss: 1.0886\n",
      "Training  ------>   Batch count: 82/549367,  batch time: 0.0411s,  batch average loss: 1.0903\n",
      "Training  ------>   Batch count: 83/549367,  batch time: 0.0369s,  batch average loss: 1.0908\n",
      "Training  ------>   Batch count: 84/549367,  batch time: 0.0387s,  batch average loss: 1.0915\n",
      "Training  ------>   Batch count: 85/549367,  batch time: 0.0384s,  batch average loss: 1.0911\n",
      "Training  ------>   Batch count: 86/549367,  batch time: 0.0369s,  batch average loss: 1.0897\n",
      "Training  ------>   Batch count: 87/549367,  batch time: 0.0393s,  batch average loss: 1.0880\n",
      "Training  ------>   Batch count: 88/549367,  batch time: 0.0400s,  batch average loss: 1.0878\n",
      "Training  ------>   Batch count: 89/549367,  batch time: 0.0369s,  batch average loss: 1.0888\n",
      "Training  ------>   Batch count: 90/549367,  batch time: 0.0373s,  batch average loss: 1.0893\n",
      "Training  ------>   Batch count: 91/549367,  batch time: 0.0404s,  batch average loss: 1.0912\n",
      "Training  ------>   Batch count: 92/549367,  batch time: 0.0395s,  batch average loss: 1.0910\n",
      "Training  ------>   Batch count: 93/549367,  batch time: 0.0406s,  batch average loss: 1.0920\n",
      "Training  ------>   Batch count: 94/549367,  batch time: 0.0376s,  batch average loss: 1.0937\n",
      "Training  ------>   Batch count: 95/549367,  batch time: 0.0370s,  batch average loss: 1.0930\n",
      "Training  ------>   Batch count: 96/549367,  batch time: 0.0418s,  batch average loss: 1.0935\n",
      "Training  ------>   Batch count: 97/549367,  batch time: 0.0371s,  batch average loss: 1.0945\n",
      "Training  ------>   Batch count: 98/549367,  batch time: 0.0364s,  batch average loss: 1.0945\n",
      "Training  ------>   Batch count: 99/549367,  batch time: 0.0396s,  batch average loss: 1.0943\n",
      "Training  ------>   Batch count: 100/549367,  batch time: 0.0427s,  batch average loss: 1.0947\n",
      "Training  ------>   Batch count: 101/549367,  batch time: 0.0369s,  batch average loss: 1.0946\n",
      "Training time: 4.1683s, loss :0.0002, accuracy: 0.0069%\n",
      "-------------------------------------------------- Validating epoch 1 --------------------------------------------------\n",
      "Testing  ------>   Batch count: 1/9842,  batch time: 0.0157s,  batch average loss: 1.0886\n",
      "Testing  ------>   Batch count: 2/9842,  batch time: 0.0130s,  batch average loss: 1.0762\n",
      "Testing  ------>   Batch count: 3/9842,  batch time: 0.0131s,  batch average loss: 1.0856\n",
      "Testing  ------>   Batch count: 4/9842,  batch time: 0.0140s,  batch average loss: 1.0792\n",
      "Testing  ------>   Batch count: 5/9842,  batch time: 0.0167s,  batch average loss: 1.0910\n",
      "Testing  ------>   Batch count: 6/9842,  batch time: 0.0145s,  batch average loss: 1.0944\n",
      "Testing  ------>   Batch count: 7/9842,  batch time: 0.0140s,  batch average loss: 1.0961\n",
      "Testing  ------>   Batch count: 8/9842,  batch time: 0.0145s,  batch average loss: 1.0950\n",
      "Testing  ------>   Batch count: 9/9842,  batch time: 0.0130s,  batch average loss: 1.0919\n",
      "Testing  ------>   Batch count: 10/9842,  batch time: 0.0133s,  batch average loss: 1.0843\n",
      "Testing  ------>   Batch count: 11/9842,  batch time: 0.0140s,  batch average loss: 1.0850\n",
      "Testing  ------>   Batch count: 12/9842,  batch time: 0.0150s,  batch average loss: 1.0945\n",
      "Testing  ------>   Batch count: 13/9842,  batch time: 0.0130s,  batch average loss: 1.0973\n",
      "Testing  ------>   Batch count: 14/9842,  batch time: 0.0121s,  batch average loss: 1.0952\n",
      "Testing  ------>   Batch count: 15/9842,  batch time: 0.0130s,  batch average loss: 1.0961\n",
      "Testing  ------>   Batch count: 16/9842,  batch time: 0.0154s,  batch average loss: 1.0954\n",
      "Testing  ------>   Batch count: 17/9842,  batch time: 0.0161s,  batch average loss: 1.0958\n",
      "Testing  ------>   Batch count: 18/9842,  batch time: 0.0130s,  batch average loss: 1.0966\n",
      "Testing  ------>   Batch count: 19/9842,  batch time: 0.0134s,  batch average loss: 1.0945\n",
      "Testing  ------>   Batch count: 20/9842,  batch time: 0.0150s,  batch average loss: 1.0951\n",
      "Testing  ------>   Batch count: 21/9842,  batch time: 0.0133s,  batch average loss: 1.0955\n",
      "Testing  ------>   Batch count: 22/9842,  batch time: 0.0126s,  batch average loss: 1.0935\n",
      "Testing  ------>   Batch count: 23/9842,  batch time: 0.0125s,  batch average loss: 1.0943\n",
      "Testing  ------>   Batch count: 24/9842,  batch time: 0.0150s,  batch average loss: 1.0946\n",
      "Testing  ------>   Batch count: 25/9842,  batch time: 0.0129s,  batch average loss: 1.0963\n",
      "Testing  ------>   Batch count: 26/9842,  batch time: 0.0129s,  batch average loss: 1.0942\n",
      "Testing  ------>   Batch count: 27/9842,  batch time: 0.0120s,  batch average loss: 1.0947\n",
      "Testing  ------>   Batch count: 28/9842,  batch time: 0.0120s,  batch average loss: 1.0951\n",
      "Testing  ------>   Batch count: 29/9842,  batch time: 0.0145s,  batch average loss: 1.0955\n",
      "Testing  ------>   Batch count: 30/9842,  batch time: 0.0140s,  batch average loss: 1.0958\n",
      "Testing  ------>   Batch count: 31/9842,  batch time: 0.0138s,  batch average loss: 1.0957\n",
      "Testing  ------>   Batch count: 32/9842,  batch time: 0.0130s,  batch average loss: 1.0958\n",
      "Testing  ------>   Batch count: 33/9842,  batch time: 0.0130s,  batch average loss: 1.0960\n",
      "Testing  ------>   Batch count: 34/9842,  batch time: 0.0142s,  batch average loss: 1.0944\n",
      "Testing  ------>   Batch count: 35/9842,  batch time: 0.0147s,  batch average loss: 1.0946\n",
      "Testing  ------>   Batch count: 36/9842,  batch time: 0.0129s,  batch average loss: 1.0964\n",
      "Testing  ------>   Batch count: 37/9842,  batch time: 0.0121s,  batch average loss: 1.0957\n",
      "Testing  ------>   Batch count: 38/9842,  batch time: 0.0130s,  batch average loss: 1.0960\n",
      "Testing  ------>   Batch count: 39/9842,  batch time: 0.0141s,  batch average loss: 1.0966\n",
      "Testing  ------>   Batch count: 40/9842,  batch time: 0.0146s,  batch average loss: 1.0970\n",
      "Testing  ------>   Batch count: 41/9842,  batch time: 0.0149s,  batch average loss: 1.0987\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing  ------>   Batch count: 42/9842,  batch time: 0.0140s,  batch average loss: 1.0991\n",
      "Testing  ------>   Batch count: 43/9842,  batch time: 0.0165s,  batch average loss: 1.0992\n",
      "Testing  ------>   Batch count: 44/9842,  batch time: 0.0116s,  batch average loss: 1.0983\n",
      "Testing  ------>   Batch count: 45/9842,  batch time: 0.0142s,  batch average loss: 1.0985\n",
      "Testing  ------>   Batch count: 46/9842,  batch time: 0.0132s,  batch average loss: 1.0987\n",
      "Testing  ------>   Batch count: 47/9842,  batch time: 0.0140s,  batch average loss: 1.0983\n",
      "Testing  ------>   Batch count: 48/9842,  batch time: 0.0130s,  batch average loss: 1.0979\n",
      "Testing  ------>   Batch count: 49/9842,  batch time: 0.0118s,  batch average loss: 1.0983\n",
      "Testing  ------>   Batch count: 50/9842,  batch time: 0.0130s,  batch average loss: 1.0982\n",
      "Testing  ------>   Batch count: 51/9842,  batch time: 0.0170s,  batch average loss: 1.0975\n",
      "Testing  ------>   Batch count: 52/9842,  batch time: 0.0128s,  batch average loss: 1.0976\n",
      "Testing  ------>   Batch count: 53/9842,  batch time: 0.0125s,  batch average loss: 1.0975\n",
      "Testing  ------>   Batch count: 54/9842,  batch time: 0.0126s,  batch average loss: 1.0976\n",
      "Testing  ------>   Batch count: 55/9842,  batch time: 0.0156s,  batch average loss: 1.0985\n",
      "Testing  ------>   Batch count: 56/9842,  batch time: 0.0159s,  batch average loss: 1.0975\n",
      "Testing  ------>   Batch count: 57/9842,  batch time: 0.0150s,  batch average loss: 1.0976\n",
      "Testing  ------>   Batch count: 58/9842,  batch time: 0.0146s,  batch average loss: 1.0972\n",
      "Testing  ------>   Batch count: 59/9842,  batch time: 0.0157s,  batch average loss: 1.0974\n",
      "Testing  ------>   Batch count: 60/9842,  batch time: 0.0149s,  batch average loss: 1.0974\n",
      "Testing  ------>   Batch count: 61/9842,  batch time: 0.0132s,  batch average loss: 1.0976\n",
      "Testing  ------>   Batch count: 62/9842,  batch time: 0.0141s,  batch average loss: 1.0981\n",
      "Testing  ------>   Batch count: 63/9842,  batch time: 0.0180s,  batch average loss: 1.0972\n",
      "Testing  ------>   Batch count: 64/9842,  batch time: 0.0140s,  batch average loss: 1.0974\n",
      "Testing  ------>   Batch count: 65/9842,  batch time: 0.0155s,  batch average loss: 1.0979\n",
      "Testing  ------>   Batch count: 66/9842,  batch time: 0.0130s,  batch average loss: 1.0974\n",
      "Testing  ------>   Batch count: 67/9842,  batch time: 0.0134s,  batch average loss: 1.0976\n",
      "Testing  ------>   Batch count: 68/9842,  batch time: 0.0140s,  batch average loss: 1.0978\n",
      "Testing  ------>   Batch count: 69/9842,  batch time: 0.0136s,  batch average loss: 1.0979\n",
      "Testing  ------>   Batch count: 70/9842,  batch time: 0.0160s,  batch average loss: 1.0976\n",
      "Testing  ------>   Batch count: 71/9842,  batch time: 0.0140s,  batch average loss: 1.0975\n",
      "Testing  ------>   Batch count: 72/9842,  batch time: 0.0140s,  batch average loss: 1.0977\n",
      "Testing  ------>   Batch count: 73/9842,  batch time: 0.0132s,  batch average loss: 1.0970\n",
      "Testing  ------>   Batch count: 74/9842,  batch time: 0.0129s,  batch average loss: 1.0976\n",
      "Testing  ------>   Batch count: 75/9842,  batch time: 0.0130s,  batch average loss: 1.0977\n",
      "Testing  ------>   Batch count: 76/9842,  batch time: 0.0130s,  batch average loss: 1.0972\n",
      "Testing  ------>   Batch count: 77/9842,  batch time: 0.0140s,  batch average loss: 1.0973\n",
      "Testing  ------>   Batch count: 78/9842,  batch time: 0.0130s,  batch average loss: 1.0983\n",
      "Testing  ------>   Batch count: 79/9842,  batch time: 0.0120s,  batch average loss: 1.0983\n",
      "Testing  ------>   Batch count: 80/9842,  batch time: 0.0130s,  batch average loss: 1.0979\n",
      "Testing  ------>   Batch count: 81/9842,  batch time: 0.0130s,  batch average loss: 1.0990\n",
      "Testing  ------>   Batch count: 82/9842,  batch time: 0.0165s,  batch average loss: 1.0985\n",
      "Testing  ------>   Batch count: 83/9842,  batch time: 0.0160s,  batch average loss: 1.0986\n",
      "Testing  ------>   Batch count: 84/9842,  batch time: 0.0138s,  batch average loss: 1.0987\n",
      "Testing  ------>   Batch count: 85/9842,  batch time: 0.0125s,  batch average loss: 1.0982\n",
      "Testing  ------>   Batch count: 86/9842,  batch time: 0.0150s,  batch average loss: 1.0983\n",
      "Testing  ------>   Batch count: 87/9842,  batch time: 0.0126s,  batch average loss: 1.0981\n",
      "Testing  ------>   Batch count: 88/9842,  batch time: 0.0131s,  batch average loss: 1.0981\n",
      "Testing  ------>   Batch count: 89/9842,  batch time: 0.0120s,  batch average loss: 1.0982\n",
      "Testing  ------>   Batch count: 90/9842,  batch time: 0.0139s,  batch average loss: 1.0982\n",
      "Testing  ------>   Batch count: 91/9842,  batch time: 0.0142s,  batch average loss: 1.0983\n",
      "Testing  ------>   Batch count: 92/9842,  batch time: 0.0130s,  batch average loss: 1.0981\n",
      "Testing  ------>   Batch count: 93/9842,  batch time: 0.0150s,  batch average loss: 1.0984\n",
      "Testing  ------>   Batch count: 94/9842,  batch time: 0.0149s,  batch average loss: 1.0985\n",
      "Testing  ------>   Batch count: 95/9842,  batch time: 0.0130s,  batch average loss: 1.0980\n",
      "Testing  ------>   Batch count: 96/9842,  batch time: 0.0131s,  batch average loss: 1.0974\n",
      "Testing  ------>   Batch count: 97/9842,  batch time: 0.0130s,  batch average loss: 1.0974\n",
      "Testing  ------>   Batch count: 98/9842,  batch time: 0.0130s,  batch average loss: 1.0969\n",
      "Testing  ------>   Batch count: 99/9842,  batch time: 0.0127s,  batch average loss: 1.0969\n",
      "Testing  ------>   Batch count: 100/9842,  batch time: 0.0130s,  batch average loss: 1.0969\n",
      "Testing  ------>   Batch count: 101/9842,  batch time: 0.0147s,  batch average loss: 1.0967\n",
      "Validating time: 1.5590s, loss: 0.0113, accuracy: 0.3556%\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#训练过程中的参数\n",
    "best_score=0.0\n",
    "train_losses=[]\n",
    "valid_losses=[]\n",
    "patience_cnt=0\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    #训练\n",
    "    print(\"-\"*50,\"Training epoch %d\"%(epoch),\"-\"*50)\n",
    "    epoch_time,epoch_loss,epoch_accuracy =train(model,train_loader,optimizer,criterion,max_grad_norm)\n",
    "    train_losses.append(epoch_loss)\n",
    "    print(\"Training time: {:.4f}s, loss :{:.4f}, accuracy: {:.4f}%\".format(epoch_time, epoch_loss, (epoch_accuracy*100)))\n",
    "    \n",
    "    #验证\n",
    "    print(\"-\"*50,\"Validating epoch %d\"%(epoch),\"-\"*50)\n",
    "    epoch_time_dev, epoch_loss_dev, epoch_accuracy_dev = validate(model,dev_loader,criterion)\n",
    "    valid_losses.append(epoch_loss_dev)\n",
    "    print(\"Validating time: {:.4f}s, loss: {:.4f}, accuracy: {:.4f}%\\n\".format(epoch_time_dev, epoch_loss_dev, (epoch_accuracy_dev*100)))\n",
    "    \n",
    "    #更新学习率\n",
    "    scheduler.step(epoch_accuracy)\n",
    "    \n",
    "    #early stoping\n",
    "    if epoch_accuracy_dev< best_score:\n",
    "        patience_cnt+=1\n",
    "    else:\n",
    "        best_score=epoch_accuracy_dev\n",
    "        patience_cnt=0\n",
    "    if patience_cnt>=patience:\n",
    "            print(\"-\"*50,\"Early stopping\",\"-\"*50)\n",
    "            break\n",
    "        \n",
    "    #每个epoch都保存模型\n",
    "    torch.save({\"epoch\": epoch,\n",
    "                \"model\": model.state_dict(),\n",
    "                \"best_score\": best_score,\n",
    "                \"train_losses\": train_losses,\n",
    "                \"valid_losses\": valid_losses},\n",
    "               model_train_dir+str(epoch)+\".dir\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "python36",
   "language": "python",
   "name": "python36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
